<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Cheaper in Bulk | Elasticsearch: The Definitive Guide [2.x] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch: The Definitive Guide [2.x]">
<link rel="up" href="data-in-data-out.html" title="Data In, Data Out">
<link rel="prev" href="_retrieving_multiple_documents.html" title="Retrieving Multiple Documents">
<link rel="next" href="distributed-docs.html" title="Distributed Document Store">
<meta name="DC.type" content="Learn/Docs/Legacy/Elasticsearch/Definitive Guide/2.x">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="2.x">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<p>
  <strong>WARNING</strong>: The 2.x versions of Elasticsearch have passed their
  <a href="https://www.elastic.co/support/eol">EOL dates</a>. If you are running
  a 2.x version, we strongly advise you to upgrade.
</p>
<p>
  This documentation is no longer maintained and may be removed. For the latest
  information, see the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html">current
  Elasticsearch documentation</a>.
</p>
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch: The Definitive Guide [2.x]</a></span>
»
<span class="breadcrumb-link"><a href="getting-started.html">Getting Started</a></span>
»
<span class="breadcrumb-link"><a href="data-in-data-out.html">Data In, Data Out</a></span>
»
<span class="breadcrumb-node">Cheaper in Bulk</span>
</div>
<div class="navheader">
<span class="prev">
<a href="_retrieving_multiple_documents.html">« Retrieving Multiple Documents</a>
</span>
<span class="next">
<a href="distributed-docs.html">Distributed Document Store »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="bulk"></a>Cheaper in Bulk<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/030_Data/55_Bulk.asciidoc">edit</a>
</h2>
</div></div></div>
<p>In the same way that <code class="literal">mget</code> allows us to retrieve multiple documents at once,
the <code class="literal">bulk</code> API allows us to make multiple <code class="literal">create</code>, <code class="literal">index</code>, <code class="literal">update</code>, or
<code class="literal">delete</code>  requests in a single step. This is particularly useful if you need
to index a data stream such as log events, which can be queued up and indexed
in batches of hundreds or thousands.</p>
<p>The <code class="literal">bulk</code> request body has the following, slightly unusual, format:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">{ action: { metadata }}\n
{ request body        }\n
{ action: { metadata }}\n
{ request body        }\n
...</pre>
</div>
<p>This format is like a <em>stream</em> of valid one-line JSON documents joined
together by newline (<code class="literal">\n</code>) characters. Two important points to note:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
Every line must end with a newline character (<code class="literal">\n</code>), <em>including the last
line</em>. These are used as markers to allow for efficient line separation.
</li>
<li class="listitem">
The lines cannot contain unescaped newline characters, as they would
interfere with parsing. This means that the JSON must <em>not</em> be
pretty-printed.
</li>
</ul>
</div>
<div class="tip admon">
<div class="icon"></div>
<div class="admon_content">
<p>In <a class="xref" href="distrib-multi-doc.html#bulk-format" title="Why the Funny Format?">Why the Funny Format?</a>, we explain why the <code class="literal">bulk</code> API uses this format.</p>
</div>
</div>
<p>The <code class="literal">action/metadata</code> line specifies <em>what action</em> to do to <em>which document</em>.</p>
<p>The <code class="literal">action</code> must be one of the following:</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
<code class="literal">create</code>
</span>
</dt>
<dd>
Create a document only if the document does not already exist. See <a class="xref" href="create-doc.html" title="Creating a New Document">Creating a New Document</a>.
</dd>
<dt>
<span class="term">
<code class="literal">index</code>
</span>
</dt>
<dd>
Create a new document or replace an existing document. See <a class="xref" href="index-doc.html" title="Indexing a Document">Indexing a Document</a> and <a class="xref" href="update-doc.html" title="Updating a Whole Document">Updating a Whole Document</a>.
</dd>
<dt>
<span class="term">
<code class="literal">update</code>
</span>
</dt>
<dd>
Do a partial update on a document. See <a class="xref" href="partial-updates.html" title="Partial Updates to Documents">Partial Updates to Documents</a>.
</dd>
<dt>
<span class="term">
<code class="literal">delete</code>
</span>
</dt>
<dd>
Delete a document. See <a class="xref" href="delete-doc.html" title="Deleting a Document">Deleting a Document</a>.
</dd>
</dl>
</div>
<p>The <code class="literal">metadata</code> should specify the <code class="literal">_index</code>, <code class="literal">_type</code>, and <code class="literal">_id</code> of the document
to be indexed, created, updated, or deleted.</p>
<p>For instance, a <code class="literal">delete</code> request could look like this:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">{ "delete": { "_index": "website", "_type": "blog", "_id": "123" }}</pre>
</div>
<p>The <code class="literal">request body</code> line consists of the document <code class="literal">_source</code> itself—​the fields
and values that the document contains.  It is required for <code class="literal">index</code> and
<code class="literal">create</code> operations, which makes sense: you must supply the document to index.</p>
<p>It is also required for <code class="literal">update</code> operations and should consist of the same
request body that you would pass to the <code class="literal">update</code> API: <code class="literal">doc</code>, <code class="literal">upsert</code>,
<code class="literal">script</code>, and so forth. No <code class="literal">request body</code> line is required for a delete.</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">{ "create":  { "_index": "website", "_type": "blog", "_id": "123" }}
{ "title":    "My first blog post" }</pre>
</div>
<p>If no <code class="literal">_id</code> is specified, an ID will be autogenerated:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">{ "index": { "_index": "website", "_type": "blog" }}
{ "title":    "My second blog post" }</pre>
</div>
<p>To put it all together, a complete <code class="literal">bulk</code> request has this form:</p>
<div class="pre_wrapper lang-sense">
<pre class="programlisting prettyprint lang-sense">POST /_bulk
{ "delete": { "_index": "website", "_type": "blog", "_id": "123" }} <a id="CO16-1"></a><i class="conum" data-value="1"></i>
{ "create": { "_index": "website", "_type": "blog", "_id": "123" }}
{ "title":    "My first blog post" }
{ "index":  { "_index": "website", "_type": "blog" }}
{ "title":    "My second blog post" }
{ "update": { "_index": "website", "_type": "blog", "_id": "123", "_retry_on_conflict" : 3} }
{ "doc" : {"title" : "My updated blog post"} } <a id="CO16-2"></a><i class="conum" data-value="2"></i></pre>
</div>
<div class="sense_widget" data-snippet="snippets/030_Data/55_Bulk.json"></div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO16-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>Notice how the <code class="literal">delete</code> action does not have a request body; it is
followed immediately by another action.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO16-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>Remember the final newline character.</p>
</td>
</tr>
</table>
</div>
<p>The Elasticsearch response contains the <code class="literal">items</code> array, which lists the result of
each request, in the same order as we requested them:</p>
<div class="pre_wrapper lang-sense">
<pre class="programlisting prettyprint lang-sense">{
   "took": 4,
   "errors": false, <a id="CO17-1"></a><i class="conum" data-value="1"></i>
   "items": [
      {  "delete": {
            "_index":   "website",
            "_type":    "blog",
            "_id":      "123",
            "_version": 2,
            "status":   200,
            "found":    true
      }},
      {  "create": {
            "_index":   "website",
            "_type":    "blog",
            "_id":      "123",
            "_version": 3,
            "status":   201
      }},
      {  "create": {
            "_index":   "website",
            "_type":    "blog",
            "_id":      "EiwfApScQiiy7TIKFxRCTw",
            "_version": 1,
            "status":   201
      }},
      {  "update": {
            "_index":   "website",
            "_type":    "blog",
            "_id":      "123",
            "_version": 4,
            "status":   200
      }}
   ]
}</pre>
</div>
<div class="sense_widget" data-snippet="snippets/030_Data/55_Bulk.json"></div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO17-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>All subrequests completed successfully.</p>
</td>
</tr>
</table>
</div>
<p>Each subrequest is executed independently, so the failure of one subrequest
won’t affect the success of the others. If any of the requests fail, the
top-level  <code class="literal">error</code> flag is set to <code class="literal">true</code> and the error details will be
reported under the relevant request:</p>
<div class="pre_wrapper lang-sense">
<pre class="programlisting prettyprint lang-sense">POST /_bulk
{ "create": { "_index": "website", "_type": "blog", "_id": "123" }}
{ "title":    "Cannot create - it already exists" }
{ "index":  { "_index": "website", "_type": "blog", "_id": "123" }}
{ "title":    "But we can update it" }</pre>
</div>
<div class="sense_widget" data-snippet="snippets/030_Data/55_Bulk_independent.json"></div>
<p>In the response, we can see that it failed to <code class="literal">create</code> document <code class="literal">123</code> because
it already exists, but the subsequent <code class="literal">index</code> request, also on document <code class="literal">123</code>,
succeeded:</p>
<div class="pre_wrapper lang-sense">
<pre class="programlisting prettyprint lang-sense">{
   "took": 3,
   "errors": true, <a id="CO18-1"></a><i class="conum" data-value="1"></i>
   "items": [
      {  "create": {
            "_index":   "website",
            "_type":    "blog",
            "_id":      "123",
            "status":   409, <a id="CO18-2"></a><i class="conum" data-value="2"></i>
            "error":    "DocumentAlreadyExistsException <a id="CO18-3"></a><i class="conum" data-value="3"></i>
                        [[website][4] [blog][123]:
                        document already exists]"
      }},
      {  "index": {
            "_index":   "website",
            "_type":    "blog",
            "_id":      "123",
            "_version": 5,
            "status":   200 <a id="CO18-4"></a><i class="conum" data-value="4"></i>
      }}
   ]
}</pre>
</div>
<div class="sense_widget" data-snippet="snippets/030_Data/55_Bulk_independent.json"></div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO18-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>One or more requests has failed.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO18-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>The HTTP status code for this request reports <code class="literal">409 CONFLICT</code>.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO18-3"><i class="conum" data-value="3"></i></a></p>
</td>
<td align="left" valign="top">
<p>The error message explaining why the request failed.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO18-4"><i class="conum" data-value="4"></i></a></p>
</td>
<td align="left" valign="top">
<p>The second request succeeded with an HTTP status code of <code class="literal">200 OK</code>.</p>
</td>
</tr>
</table>
</div>
<p>That also means that <code class="literal">bulk</code> requests are not atomic: they cannot be used to
implement transactions.  Each request is processed separately, so the success
or failure of one request will not interfere with the others.</p>
<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="_dont_repeat_yourself"></a>Don’t Repeat Yourself<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/030_Data/55_Bulk.asciidoc">edit</a>
</h3>
</div></div></div>
<p>Perhaps you are batch-indexing logging data into the same <code class="literal">index</code>, and with the
same <code class="literal">type</code>. Having to specify the same metadata for every document is a waste.
Instead, just as for the <code class="literal">mget</code> API, the <code class="literal">bulk</code> request accepts a default <code class="literal">/_index</code> or
<code class="literal">/_index/_type</code> in the URL:</p>
<div class="pre_wrapper lang-sense">
<pre class="programlisting prettyprint lang-sense">POST /website/_bulk
{ "index": { "_type": "log" }}
{ "event": "User logged in" }</pre>
</div>
<div class="sense_widget" data-snippet="snippets/030_Data/55_Bulk_defaults.json"></div>
<p>You can still override the <code class="literal">_index</code> and <code class="literal">_type</code> in the metadata line, but it
will use the values in the URL as defaults:</p>
<div class="pre_wrapper lang-sense">
<pre class="programlisting prettyprint lang-sense">POST /website/log/_bulk
{ "index": {}}
{ "event": "User logged in" }
{ "index": { "_type": "blog" }}
{ "title": "Overriding the default type" }</pre>
</div>
<div class="sense_widget" data-snippet="snippets/030_Data/55_Bulk_defaults.json"></div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="_how_big_is_too_big"></a>How Big Is Too Big?<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/030_Data/55_Bulk.asciidoc">edit</a>
</h3>
</div></div></div>
<p>The entire bulk request needs to be loaded into memory by the node that
receives our request, so the bigger the request, the less memory available for
other requests. There is an optimal size of bulk request. Above that size,
performance no longer improves and may even drop off. The optimal size, however, is not a fixed number. It depends entirely on your
hardware, your document size and complexity, and your indexing and search
load.</p>
<p>Fortunately, it is easy to find this <em>sweet spot</em>: Try indexing typical documents in batches of increasing size. When performance
starts to drop off, your batch size is too big. A good place to start is with
batches of 1,000 to 5,000 documents or, if your documents are very
large, with even smaller batches.</p>
<p>It is often useful to keep an eye on the physical size of your bulk requests.
One thousand 1KB documents is very different from one thousand 1MB documents.
A good bulk size to start playing with is around 5-15MB in size.</p>
</div>

</div>
<div class="navfooter">
<span class="prev">
<a href="_retrieving_multiple_documents.html">« Retrieving Multiple Documents</a>
</span>
<span class="next">
<a href="distributed-docs.html">Distributed Document Store »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
